import time

import requests
from bs4 import BeautifulSoup
import re
from multiprocessing.dummy import Pool as ThreadPool

from twisted.python import threadpool


def saveImg(img):
    Note = open(r'D:\CodeReference\img\123.txt', mode='a')
    for j in img:
        Note.write(j.replace("scpic3", 'scpic').replace("_s.jpg", ".jpg") + '\n')
    Note.close()


# url = 'https://sc.chinaz.com/tupian/renwutupian_2.html'




def getImgUrl(url):
    import requests
    # url = "https://sc.chinaz.com/tupian/renwutupian_" + str(i) + ".html"
    print(url)
    headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
               'Accept-Encoding': 'gzip, deflate, br',
               'Accept-Language': 'zh-CN,zh;q=0.9',
               'Cache-Control': 'max-age=0',
               'Connection' : 'keep-alive',
               'Cookie': 'sctj_uid=c73d2cde-3d7e-9ee7-6aae-62fe48b34248; Hm_lvt_398913ed58c9e7dfe9695953fb7b6799=1661522757,1661524878,1661575547; Hm_lpvt_398913ed58c9e7dfe9695953fb7b6799=1661578294',
              'Host' : 'sc.chinaz.com',
               'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36'}

    res = requests.get(url, headers=headers)

    soup = BeautifulSoup(res.text, "html.parser")
    divTag = soup.find_all('img', 'lazy')

    img = re.findall('data-original="(.*?)"', str(divTag), re.S)
    #

    print(img)
    saveImg(img)


# 网站
list1 = []
for i in range(2, 101):
    url = "https://sc.chinaz.com/tupian/renwutupian_" + str(i) + ".html"
    list1.append(url)


start_time = time.time()
# 第一步 创建线程池，线程数为10：
pool = ThreadPool(processes=10)
# 第二步创建线程请求，包涵调用的函数、参数和回调函数：
# requests = pool.makeRequests(func, args_list, call_back)

results = pool.map(getImgUrl, list1)  # 需要执行的方法和地址列表
pool.close()
pool.join()
print('用时共： %s second' % (time.time() - start_time))
